爬虫

2024-04-02 23:15| 来源: 网络整理| 查看: 265

接上一篇，get请求的url的参数是直接被暴露在url地址中的，而且有字数限制；而post请求的url的参数并不能直接进行拼接，而是放在data中进行发送，无字数限制，直观的url地址中不会显示post请求的参数。

一、简单翻译

在对百度翻译进行post请求时需要找到自己需要的数据，【F12--network--sug--preview；preview里的kw就是post请求发送的参数，放在data里】

post请求必须进行编码【urlencode.encode('utf-8')】,也就是二次编码。

data = urllib.parse.urlencode(data).encode('utf-8')

因为post请求返回的数据是字节形式的数据，所以我们要进行反序列化【爬虫_03_序列化与反序列化_唯念欢的博客-CSDN博客】

json.loads方法

完整代码如下。

import urllib.request import urllib.parse import json url = 'https://fanyi.baidu.com/sug' data = { 'kw': 'spider' } headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 Edg/114.0.1823.67' } data = urllib.parse.urlencode(data).encode('utf-8') request = urllib.request.Request(url=url, data=data, headers=headers) response = urllib.request.urlopen(request) content = response.read().decode('utf-8') obj = json.loads(content) print(obj)

二、详细翻译【v2transapi]

这里介绍pycharm的正则表达式替换：

查找：(.?*):(.*)

替换：'$1':'$2',

在详细翻译中会遇到反爬手段，这是因为请求头中的数据没有给全，这时候需要将请求头的数据贴给headers【但是其中某些数据是一定要注释的’Accept-Encoding: gzip, deflate, br‘】

其中起决定性作用的是headers中的cookie。

import urllib.request import urllib.parse import json url = 'https://fanyi.baidu.com/sug' data = { 'from':'en', 'to':'zh', 'query':'love', 'transtype':'translang', 'simple_means_flag':'3', 'sign':'198772.518981', 'token':'75cd025e04a5a019fb1b9db63a65af40', 'domain':'common', 'ts':'1688566616188' } headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 Edg/114.0.1823.67' } data = urllib.parse.urlencode(data).encode('utf-8') request = urllib.request.Request(url=url, data=data, headers=headers) response = urllib.request.urlopen(request) content = response.read().decode('utf-8') obj = json.loads(content) print(obj) # {'errno': 1000, 'errmsg': '未知错误'}

在这里我遇到了一个非常经典的错误，就是用pycharm进行正则表达式替换时，可能data和headers的单引号内会有空格，此时会返回未知错误，在这里提出来用以给我进行惊醒和给各位小伙伴进行提示

改进后。

import urllib.request import urllib.parse import json url = 'https://fanyi.baidu.com/v2transapi?from=en&to=zh' data = { 'from':'en', 'to':'zh', 'query':'love', 'transtype':'translang', 'simple_means_flag':'3', 'sign':'198772.518981', 'token':'75cd025e04a5a019fb1b9db63a65af40', 'domain':'common', 'ts':'1688566616188' } headers = { 'Accept':'*/*', # 'Accept-Encoding':' gzip, deflate, br', 'Accept-Language':'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6', 'Acs-Token':'1688566609428_1688566616202_9fHjVbZcxo4pLKO6A8cKGj84jE94Alu0wI8oO/pKahcvgYb4ULBKb8R6Ts4D4APo2unrmlLnkLz/42q2qqsR1T6PFIIVtGvxMterxxDhYVFDef3vN3/bPj0cVZp1pL3gi8pSf10k/8AOhIzgpJlriV/RTiTUA6qgJpp6Thj6niCBCQxN8r19bD1c/UYClX/X0ccJdnIKGQasvxNPIujQaCah+FA5XLy5JoUIERMeR6ETsQ7qEqSKZ6ExRTrQ4vKWZH+bim0JI98efKX60e6Z8un2fhnuKkCjOrPFvO0zL6igjsFC5z+0acM6DJnh8FjnFsZw4KNTdYNWH+ohc5Rv5OkmL9ayz86yQQXOdZzcTPiycyLdFtm/vudjUF/LXjofZgYwuLcVeNXWx76ceHGs2EQqHixCTh0LHg3+pXkCnCVLJR8OlfUOV5rue3UGkz79Uo+erA2afClifCyHYqFYTKG5hh8gixopoqn6vHFOBLISnEwfnh8aLGA/ZeKd9fe4nVF5PtPIgf8aEzGK64uxfA==', 'Connection':'keep-alive', 'Content-Length':'135', 'Content-Type':'application/x-www-form-urlencoded; charset=UTF-8', 'Cookie':'H_WISE_SIDS=110085_176399_180636_188749_194529_204912_211986_212295_213033_213353_214792_215730_216844_216942_219556_219623_219942_219946_222624_223064_224045_224047_226600_226628_226956_227932_228221_228650_229154_229905_229967_230083_230240_230287_230930_231418_231430_231628_231760_231904_231919_231959_231979_232245_232281_232476_232550_232728_232777_232908_233368_233466_233518_233604_234020_234045_234085_234288_234316_234515_234521_234560_234687_234725_234798_234922_234953_234958_235090_235181_235201_235398_235417_235438_235443_235485_235534_235583_235744_235832_235980_236052_236084_236103_236237_236243_236396_236511_236517_236521_236530_236540_236610_236785_236810_236940_237113_237135_237230_237240_237251_237253_237255_237293_237312_237392_237448_237450_237644_237656_237700_237806; BAIDUID=BA77F27FA293A82ECF199DE64CC3103B:FG=1; BDUSS=35wVjN2MFBuNmJlU0pmM1QwcHlpSmFLOHFZb0l6WkhLbXlNOVRGc1RGMi1UclZrSVFBQUFBJCQAAAAAAQAAAAEAAAACfmV4x-XT8MSrsLK7tgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAL7BjWS-wY1kS; BDUSS_BFESS=35wVjN2MFBuNmJlU0pmM1QwcHlpSmFLOHFZb0l6WkhLbXlNOVRGc1RGMi1UclZrSVFBQUFBJCQAAAAAAQAAAAEAAAACfmV4x-XT8MSrsLK7tgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAL7BjWS-wY1kS; BIDUPSID=BA77F27FA293A82ECF199DE64CC3103B; PSTM=1687169997; REALTIME_TRANS_SWITCH=1; FANYI_WORD_SWITCH=1; HISTORY_SWITCH=1; SOUND_SPD_SWITCH=1; SOUND_PREFER_SWITCH=1; H_WISE_SIDS_BFESS=110085_176399_180636_188749_194529_204912_211986_212295_213033_213353_214792_215730_216844_216942_219556_219623_219942_219946_222624_223064_224045_224047_226600_226628_226956_227932_228221_228650_229154_229905_229967_230083_230240_230287_230930_231418_231430_231628_231760_231904_231919_231959_231979_232245_232281_232476_232550_232728_232777_232908_233368_233466_233518_233604_234020_234045_234085_234288_234316_234515_234521_234560_234687_234725_234798_234922_234953_234958_235090_235181_235201_235398_235417_235438_235443_235485_235534_235583_235744_235832_235980_236052_236084_236103_236237_236243_236396_236511_236517_236521_236530_236540_236610_236785_236810_236940_237113_237135_237230_237240_237251_237253_237255_237293_237312_237392_237448_237450_237644_237656_237700_237806; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; BDRCVFR[feWj1Vr5u3D]=I67x6TjHwwYf0; delPer=0; PSINO=3; H_PS_PSSID=; BA_HECTOR=akag208l2h0hak0hala0ah0a1iaaqm41p; ZFY=i6g:BfYiyBJbM4MMGMSuLqQ45Y3kAxlUKWhmlUSpzR:BA:C; BAIDUID_BFESS=BA77F27FA293A82ECF199DE64CC3103B:FG=1; Hm_lvt_64ecd82404c51e03dc91cb9e8c025574=1687430571,1688562907; Hm_lpvt_64ecd82404c51e03dc91cb9e8c025574=1688566609; ab_sr=1.0.1_N2YzZTAxYzhkYTNiMTdlYjIzODI0YTMwYjVjZGY0OGVmZTNjM2Y2ZGYxN2ViM2JiOTViMDEzMzc3ZWI0NDA4ZTMwOGY3MGZiOTI0Mjg5ODllMmU4YWMxNjY1MDNmMTIxMWMzMzdkYjVjMTM1N2JiNDE5ZTMzMzZiNzVhZjc4ZjU2ZTYzM2Y2NTQxODYxNDczMzYzYWQ2MjRjNjUxYzFiMjQyZGQ2Njg2NWUxZjg2OGNjMTkzNzU0ZjhiOTY1NzVl', 'Host':'fanyi.baidu.com', 'Origin':'https://fanyi.baidu.com', 'Referer':'https://fanyi.baidu.com/', 'Sec-Fetch-Dest':'empty', 'Sec-Fetch-Mode':'cors', 'Sec-Fetch-Site':'same-origin', 'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 Edg/114.0.1823.67', 'X-Requested-With':'XMLHttpRequest', 'sec-ch-ua':'"Not.A/Brand";v="8", "Chromium";v="114", "Microsoft Edge";v="114"', 'sec-ch-ua-mobile':'?0', 'sec-ch-ua-platform': '"Windows"' } data = urllib.parse.urlencode(data).encode('utf-8') request = urllib.request.Request(url=url, data=data, headers=headers) response = urllib.request.urlopen(request) content = response.read().decode('utf-8') obj = json.loads(content) print(obj) # {'trans_result': {'data': [{'dst': '爱', 'src': 'love'}], 'from': 'en', 'to': 'zh', 'status': 0······

【本文地址】

公司简介

联系我们